import re

from time import time
from zope.interface import implements
from twisted.internet import defer, reactor, protocol
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import IBodyProducer
from twisted.internet.error import TimeoutError
from twisted.web.http import PotentialDataLoss
from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, HTTPConnectionPool, TCP4ClientEndpoint
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
from scrapy.core.downloader.tls import openssl_methods
from scrapy.utils.misc import load_object
from scrapy.http import HtmlResponse
from twisted.internet import utils


class MyDownloadHandler(object):
    """
    自定义下载器接口,被上层所调用
    """
    def __init__(self, settings):
        self._pool = HTTPConnectionPool(reactor, persistent=True)
        self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
        self._pool._factory.noisy = False
        self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
        self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
        try:
            self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
        except TypeError:
            # use context factory defaults
            self._contextFactory = self._contextFactoryClass()
            msg = """
            '%s' does not accept `method` argument (type OpenSSL.SSL method,\
            e.g. OpenSSL.SSL.SSLv23_METHOD).\
            Please upgrade your context factory class to handle it or ignore it.""" % (settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
            # warnings.warn(msg)

    def download_request(self, request, spider):
        """Return a deferred for the HTTP download"""
        selenium_downloader = SeleniumDownloader()
        return selenium_downloader.download(request)

    def close(self):
        return self._pool.closeCachedConnections()


class SeleniumDownloader(object):
    """selenium下载器"""
    def __init__(self, agent=None):
        '''agent: 异步下载代理'''
        self._agent = agent

    def download(self, request):
        '''
        需要异步返回，不可以阻塞，本例子的演示直接调用 phantomjs的一个简单包装脚本
	    '''
        begintime = time()
        # d = self._download(request)
        d = utils.getProcessOutput('scrapyweb.js', args=(request.url, '24000'), reactor=reactor)
        d.addCallback(lambda result: result)
        d.addCallback(self.parseData, request, begintime)
        print('证明我是异步的')
        return d

    # def _download(self, request):
    #     '''使用twsited 的函数创建异步进程调用'''
    #     d = utils.getProcessOutput('scrapyweb.js', args=(request.url, '24000'), reactor=reactor)
    #     # def getOutput(result):
    #     #     return result
    #     d.addCallback(lambda result:result)
    #     return d

    def parseData(self, htmldoc, request, begintime):
        '''解析函数，当请求完成后被调用'''
        # 这个下载时间在调整下载速度的扩展 AutoThrottle 中被使用
        request.meta['download_latency'] = time() - begintime
        return HtmlResponse(request.url, body=htmldoc + '\n证明我被解析过', request=request)

